#1.固定路径导入
#2.动态路径导入 用os，sys,__file__进行动态导入，需要在所有文件加上动态导入
#3.通过入口程序对所有文件进行连接可以导入


#放在第三方模块里


# import json,time,os,hashlib
# def laod(name,expire_date = None,status = None,password = None,all = None):
#     f = open(name + '.json', 'r')
#     j_a = json.load(f)
#     a = json.loads(j_a)
#     f.close()
#     if expire_date == 1:
#         return a['expire_date']
#     elif status == 1:
#         return a['status']
#     elif password == 1:
#         return a['password']
#     elif all == 1:
#         return a
# def save_back(name,status = None,password =None):
#     a = laod(name,all= 1)           #a = {'expire_date': '2021-01-01', 'id': 1234, 'status': 0, 'pay_day': 22, 'password': 'abc'}
#     if status == 1:
#         a['status'] = '1'
#     elif password:
#         a['password'] = password
#     f = open(name + '.json', 'w')
#     f.seek(0)
#     f.truncate()
#     json.dump(json.dumps(a),f)
#     f.clouse()
# def change_pasd(name):
#     pasd = laod(name,password= 1)
#     md5_pasd = hashlib.md5()
#     md5_pasd.update(pasd.encode('utf-8'))
#     md5_pasd = md5_pasd.hexdigest()             #900150983cd24fb0d6963f7d28e17f72
#     save_back(name,password='900150983cd24fb0d6963f7d28e17f72')
# def login():
#     while True:
#         name = 'acconut'#input('name:')
#         if os.path.isfile(os.getcwd() +'\\' + name + '.json') == True:
#             conut = 0
#             user_pasd = laod(name, password= 1)            #user_pasd = abc
#             while True:
#                 pasd = 'abc'#input('password:')
#                 m = hashlib.md5()
#                 m.update(pasd.encode('utf-8'))
#                 pasd = m.hexdigest()
#                 if pasd != user_pasd:
#                     conut += 1
#                     print('password is wrong!')
#                     if conut == 3:
#                         save_back(name,status= 1)        #status改为1 并存入数据 退出程序
#                         print('登录失败，账户已被锁定')
#                 else:
#                     break
#             if laod(name,status= 1) == '0':                 #laod(name,status= 1) = 0
#                 now_time = time.time()     #计算时间用时间戳比较time.time()
#                 a_time = time.mktime(time.strptime(laod(name,expire_date=1),'%Y-%m-%d'))
#                 if a_time >= now_time:
#                     print('登陆成功')
#                     break
#                 else:
#                     print('账户已过期')
#             else:
#                 print('账户已被锁定不能登录')
#         elif name == 'q':
#             exit('程序已退出')
#         else:
#             print('canot you name')
#             continue
# login()
# change_pasd()



# import random,string
# a = ' '.join(random.sample(string.ascii_uppercase+string.digits,5))
# print('\033[0;35;42m%s\033[0m'% a)



# import json,xlwt
# def load():
#     f = open(r'C:\Users\Administrator\Desktop\33333.txt','r',encoding='utf-8')
#     a = f.read().encode('utf-8')
#     a = json.loads(a)
#     return a
# def save_excal(a,excal_name):
#     wb = xlwt.Workbook()           #读取数据
#     ws = wb.add_sheet('student',cell_overwrite_ok=True)
#     row = 0
#     col = 0
#     for k,v in sorted(a.items(),key=lambda d:d[0]):
#         ws.write(row,col,k)
#         for i in v:
#             col += 1
#             ws.write(row,col,i)
#         row +=1
#         col = 0
#     wb.save(excal_name)
# a = load()
# print(a)
# save_excal(a,'111.xls')


# import json,xlwt
# f = open(r'C:\Users\Administrator\Desktop\33333.txt','r',encoding='utf-8')
# a = f.read().encode('utf-8')
# a = json.loads(a)
# f.close()
# file = xlwt.Workbook()
# table = file.add_sheet('test')
# for row,i in enumerate(list(a)):
#     table.write(row,0,i)
#     for col,j in enumerate(a[i]):
#         table.write(row,col +1,j)
# file.save('111.xlsx')




#filename = #日志文件
import re,datetime,pandas,xlwt
obj = re.compile(r'(?P<ip>.*?)--\[(?P<time>.*?)\]"(?P<request>.*?)"(?P<status>)(?P<bytes>.*?)"(?P<referer>.*?)""(?P<ua>.*?)"')
#将一行的内容用re模块去正确匹配的规定
def parse(line):                        #对一行内容进行正则匹配，进行数据处理
    dic = {}
    try:
        result = obj.match(line)
        #IP处理
        ip = result.group('ip')
        if ip.strip() == '-' or ip.strip() == '':
            return False
        dic['ip'] =ip.strip(',')[0]
        #状态码处理
        status = result.group('status')
        dic['status'] =status
        #时间处理
        tim = result.group('time')
        tim = tim.replace(' +0800','')
        t = datetime.datetime.strptime(tim,'%d/%m/%Y:%H:%M%s')
        dic['time'] = t
        #request
        request = result.group('request')
        a = request.split()[1].split('?')[0]
        dic['request'] = a
        #user_ahent
        ua = result.group('ua')
        if 'Windows NT' in ua:
            u = 'windows'
        elif 'ipad' in ua:
            u = 'ipad'
        elif 'Android' in ua:
            u = 'android'
        elif 'Macintoch' in ua:
            u = 'mac'
        elif 'iphone' in ua:
            u = 'iphone'
        else:
            u = '其他设备'
        dic['ua'] = u
        #refer
        referer = result.group('referer')
        dic['referer'] = referer
        return dic
    except:
        return False

def load(filname):
    conut = 0
    with open(filname,'r',encoding='utf-8') as f:    #读取日志内容
        for i in f:
            i = i.strip()                  #清除一行中的空格
            dic = parse(i)                #应用parse函数
            if dic:
                lst.append(dic)            #将每一个字典加入lst列表
            else:
                error_lst.append(i)          #将错误日志加入到error——lst列表
            conut += 1
    print(i)                             #总日志行数
    print(len(lst))                          #正确行数
    print(len(error_lst))                   #错误行数
def analyes(lst):
    df = pandas.DataFrame(lst)     #读取列表内容到df转变成表格类型
    ip_conut = pandas.value_counts(df['ip']).reset_index().rename(columns={'index':'ip','ip':'conut'}).iloc[:20, :]
    #显示top20的ip   读取df里ip的数值进行整理输出IP的出现次数
    ip_conut_val = ip_conut.values      #将pandas处理后的数据转变成普通数据
    #写入xls文件
    wb = xlwt.Workbook()                    #打开一个excel文档
    sheet = wb.add_sheet('top 20 ip')           #新建一个sheet页
    row = 0
    sheet.write(row,0,'ip')
    sheet.write(row,1,'conut')
    row += 1
    for i in ip_conut_val:
        sheet.write(row, 0, i[0])
        sheet.write(row, 1, i[1])
        row += 1
    wb.save('111.xls')

if __name__ == '__main__':
    lst = []
    error_lst = []
    lst,error_lst = load('aa.log')
    analyes(lst)


















